    <?php 
    /**
    * All rights reserved.
    *
    * Redistribution and use in source and binary forms, with or without modification, are permitted provided
    * that the following conditions are met:
    *
    *    * Redistributions of source code must retain the above copyright notice, this list of conditions and 
    *       the following disclaimer.
    *    * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and
    *       the following disclaimer in the documentation and/or other materials provided with the distribution.
    *    * Neither the name of the Art-k-tec Systems, LLC nor the names of its contributors may be used to endorse
    *       or promote products derived from this software without specific prior written permission.
    *
    * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
    * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
    * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
    * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
    * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
    * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
    * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
    * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
    * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
    * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
    * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
    *
    *  @link http://www.art-k-tec.com
    *  @since 0.1
    *  @version 1.0
    *  @author Nika Jones <njones@art-k-tec.com>
    *  @copyright 2007 Art-k-tec Systems, LLC
    *  @license The New BSD Licence
    */

    define('MAX_RECURSION', 2);

    class LinkException extends Exception {
        const BAD_RESOURCE              = 1;
        const JAVASCRIPT_RESOURCE       = 2;
    }

    class Link {

        const MAX_RECURSION             = MAX_RECURSION;
        const LANGUAGE_REPORT           = 'en';

        const REPORT_AQUIRED_FILE       = '/report/loaded/file';
        const REPORT_LOADED_DOM         = '/report/loaded/form';
        const REPORT_LOADED_SIMPLEXML   = '/report/loaded/simplexml';
        const REPORT_LINK_LIST          = '/report/link/list';
        const REPORT_ADDED_CHILD_LINK   = '/report/link/add';
        const REPORT_REJECT_CHILD_LINK  = '/report/link/reject/';
        const REPORT_LINK_BAD           = '/report/link/bad';
        const REPORT_QUEUE_COUNT        = '/report/queue/count';
        const REPORT_RECURSION_LEVEL    = '/report/recursion/level';

        const PARSE_META_TAG            = '|PARSE[MT]|';

        const NO_QUERYSTRING            = 1;
        const DOMAIN_NONE               = 2;
        const DOMAIN_STRICT             = 4;
        const SUBDOMAIN_STRICT          = 8;

        private $sLinkParent;
        private $sLink;
        private $sPage;
        private $oDom;
        private $nLevel;

        private $aRule;

        public function __construct($sLinkParent, $sLink, $nLevel=null) {
            $aParameter = func_get_args();
            
            $this->aRule = count($aParameter) > 3 ? array_slice($aParameter, 3) : array();
            $this->sLinkParent = $sLinkParent;
            $this->sLink = $sLink;
            $this->nLevel = ($sLink ==$sLinkParent ? 0 : (int)$nLevel);
        }

        public function Link() {
            return $this->sLink;
        }

        public function Crawl($cCrawlFlag=null) {
            if(is_null($cCrawlFlag)) { $cCrawlFlag = Link::NO_QUERYSTRING | Link::DOMAIN_STRICT; }
            if($this->nLevel == Link::MAX_RECURSION) { return; }

            $oLinkQueue = LinkQueue::GetInstance();
            $nLevel = ($this->nLevel + 1);

            $this->Report(Link::REPORT_QUEUE_COUNT, $oLinkQueue->Count());
            $this->Report(Link::REPORT_RECURSION_LEVEL, $this->nLevel);

            try {
                $this->LoadPage();

                $oDom = @DomDocument::LoadHtml($this->sPage);
                if($oDom === false) {
                    throw new LinkException('DOM could not load the HTML file. Please Check the html file');
                }

                $this->oDom = $oDom;
                $this->Report(Link::REPORT_LOADED_DOM);

                $xPage = simplexml_import_dom($oDom);
                $this->Report(Link::REPORT_LOADED_SIMPLEXML);

                $aLink = $xPage->xpath('//a/@href[not(contains(., "mailto"))]');
                $this->Report(Link::REPORT_LINK_LIST, $aLink);

                $bParent = preg_match_all('/(https?:\/\/)?(.[^\/]+\/?)?(.+\/?)*/', $this->sLinkParent, $aLinkParent);
                if(!$bParent) {
                    throw new LinkException('The parent link was a completely invalid link.');
                }

                $sLinkParentProtocal    = $aLinkParent[1][0];
                $sLinkParentDomain      = $aLinkParent[2][0];
                $sLinkParentPath        = $aLinkParent[3][0];

                foreach($aLink as $sLinkChild) {
                    $sLinkChild = (string)$sLinkChild;

                    if($cCrawlFlag & Link::NO_QUERYSTRING) {
                        $nQueryStringStart = (strpos($sLinkChild, '?') ? strpos($sLinkChild, '?') : strlen($sLinkChild));
                        $sLinkChild = substr($sLinkChild, 0, $nQueryStringStart);
                    }

                    $bChild = preg_match_all('/(https?:\/\/)?(.[^\/]+\/?)?(.+\/?)*/', $sLinkChild, $aLinkChild);
                    if(!$bChild) {
                        throw new LinkException('The child link was a completely invalid link.');
                    }
                    $sLinkChildProtocal     = $aLinkChild[1][0];
                    $sLinkChildDomain       = implode('', explode('/', $aLinkChild[2][0]));
                    $sLinkChildPath         = $aLinkChild[3][0];

                    if(eregi('javascript:', $sLinkChildPath)) {
                        throw new LinkException('Can not index Javascript Links', LinkException::JAVASCRIPT_RESOURCE);
                    }

                    switch(true) {

                        case ($cCrawlFlag & Link::DOMAIN_NONE):
                            $oLink = new Link($this->sLink, $sLinkChild, $nLevel);
                            $oLinkQueue->Enqueue($oLink);
                            $this->Report(Link::REPORT_ADDED_CHILD_LINK, $sLinkChild);
                            $this->Report(Link::REPORT_QUEUE_COUNT, $oLinkQueue->Count());
                        break;

                        case ($cCrawlFlag & Link::DOMAIN_STRICT):

//                          echo "<span style='color:red'>$sLinkChildProtocal && $sLinkChildPath && ($sLinkChildDomain != $sLinkParentDomain)</span><br />";
                            if(empty($sLinkChildProtocal) && empty($sLinkChildPath) && ($sLinkChildDomain != $sLinkParentDomain)) {
                                $sLinkChildPath = $sLinkChildDomain;
                                $sLinkChildDomain = $sLinkParentDomain;
                            }

//                          echo "<span style='color:purple'>domain:$sLinkChildDomain path: $sLinkChildPath</span><br />";
                            $aDomainChild = explode('.', $sLinkChildDomain);
                            $aDomainParent = explode('.', $sLinkParentDomain);

                            $sDomainChild = array_pop($aDomainChild);
                            $sDomainChild = array_pop($aDomainChild).'.'.$sDomainChild;
                            $sDomainParent = array_pop($aDomainParent);
                            $sDomainParent = array_pop($aDomainParent).'.'.$sDomainParent;

//                          echo "<span style='color:green'>$sDomainChild == $sDomainParent</span><br />";
                            if($sDomainChild == $sDomainParent) {
                                if(empty($sLinkProtocal)) {
                                    $sLinkChildProtocal = $sLinkParentProtocal;
                                }

                                $sTrailSlash = $sLinkChildDomain[strlen($sLinkChildDomain)-1];
                                $sLeadSlash = (empty($sLinkChildPath) ? '' : $sLinkChildPath[0]);

                                $sLinkChild = 
                                $sLinkChildProtocal.
                                $sLinkChildDomain.
                                (($sTrailSlash == '/' || $sLeadSlash == '/') ? '' : '/').
                                $sLinkChildPath;

//                              echo "<span style='color:brown;font-weight:bold'>$sLinkChild</span><br />";
                                $oLink = new Link($this->sLink, $sLinkChild, $nLevel);
                                if($oLinkQueue->Enqueue($oLink)) {
                                    $this->Report(Link::REPORT_ADDED_CHILD_LINK, $sLinkChild);
                                    $this->Report(Link::REPORT_QUEUE_COUNT, $oLinkQueue->Count());
                                }
                            } else {
                                $this->Report(Link::REPORT_REJECT_CHILD_LINK, $sLinkChild);
                            }
                        break;

                        case ($cCrawlFlag & (Link::SUBDOMAIN_STRICT | Link::DOMAIN_STRICT)):
                            if($sLinkDomain == $aLinkParent[2]) {
                                $oLink = new Link($this->sLink, $sLinkChild, $nLevel);
                                $oLinkQueue->Enqueue($oLink);
                                $this->Report(Link::REPORT_ADDED_CHILD_LINK, $sLinkChild);
                                $this->Report(Link::REPORT_QUEUE_COUNT, $oLinkQueue->Count());
                            } else {
                                $this->Report(Link::REPORT_REJECT_CHILD_LINK, $sLinkChild);
                            }
                        break;
                        default:
                            $this->Report(Link::REPORT_REJECT_CHILD_LINK, $sLinkChild);
                        break;
                    }
                }
            } catch(LinkException $oLinkException) {
                $nBadResources = LinkException::BAD_RESOURCE | LinkException::JAVASCRIPT_RESOURCE;
                if($oLinkException->getCode() & $nBadResources) {
                    $oLinkQueue->BadResource($this);
                    $this->Report(Link::REPORT_LINK_BAD, $this);
                }
            }
        }

        public function Parse($cParseType=null) {
            switch($cParseType) {
                case Link::PARSE_META_TAG:
                    $oReturn = simplexml_import_dom($this->oDom);
                    return $oReturn->xpath('//meta | //META ');
                break;
            }
        }

        private function Report($cReportType, $mData=null) {
            switch($cReportType) {
                case Link::REPORT_AQUIRED_FILE:
                    $sFlush = '<br/><br/>Loaded in the file: '.$mData.'.';
                break;
                case Link::REPORT_LOADED_DOM:
                    $sFlush = 'Loaded the HTML to DOM.';
                break;
                case Link::REPORT_LOADED_SIMPLEXML:
                    $sFlush = 'Loaded the SimpleXML Object.';
                break;
                case Link::REPORT_LINK_LIST:
                    $sFlush = 'The Link List has be loaded. '.count($mData).' link(s) have been counted.';
                break;
                case Link::REPORT_ADDED_CHILD_LINK:
                    $sFlush = 'Added the child link '.$mData.' to the link queue.';
                break;
                case Link::REPORT_QUEUE_COUNT:
                    $sFlush = 'There are '.$mData.' links in the queue.';
                break;
                case Link::REPORT_LINK_BAD:
                    $sFlush = 'That was a BAD LINK. <span style="color:red">'.$mData->Link()."</span><br />Link NOT ADDED";
                break;
                case Link::REPORT_RECURSION_LEVEL:
                    $sFlush = "The RECURSION is at level ".$mData;
                break;
                case Link::REPORT_REJECT_CHILD_LINK:
                    $sFlush = "The following link was NOT added to the Queue: <b>".$mData."</b>";
                break;
            }

            echo $sFlush."<br />";
            ob_flush();
            flush();
        }

        private function LoadPage() {
            $aContext = array('http' => array ('user-agent'=>'osCrawl'));
            $oContext = stream_context_create($aContext);
            $this->sPage = @file_get_contents($this->sLink, FALSE, $oContext);

            if($this->sPage === false) {
                throw new LinkException('This is a bad resource.', LinkException::BAD_RESOURCE);
            }

            $this->Report(Link::REPORT_AQUIRED_FILE, $this->sLink);
        }
    }

    class LinkQueue extends ArrayObject {
        static private $oQueue;
        private $aQueue;
        private $aProcessed;
        private $aBadLink;

        public function GetInstance() {
            if(is_null(self::$oQueue)) { 
                self::$oQueue = new LinkQueue();
            }
            return self::$oQueue;
        }

        public function Count() {
            $oIterator = $this->getIterator();
            return count($oIterator);
        }

        public function BadResource(Link $oLink) {
            $sHash = md5($oLink->Link());
            $this->aProcessed[$sHash] = true;
            $this->aBadLink[$sHash] = true;
        }

        public function Enqueue(Link $oLink) {
            $sHash = md5($oLink->Link());
            if(!isset($this->aProcessed[$sHash])) {
                $oIterator = $this->getIterator();
                $oIterator->offsetSet($sHash, $oLink);
                $this->aProcessed[$sHash] = true;
                return true;
            } else {
                return false;
            }
        }

        public function Dequeue() {
            $oIterator = $this->getIterator();
            if($oIterator->valid()) {
                $oIterator->rewind();
                $oReturn = $oIterator->current();
                $oIterator->offsetUnset($oIterator->key());
                return $oReturn;
            } else {
                return false;
            }
        }
    }



    class LinkSchedule {

        public function __construct($nMaxRecursion=null, $nMaxChildProcess=1){}

        public function start($sLinkParent, $nMaxRecursionDepth=null) {
            if(is_null($nMaxRecursionDepth)) { $nMaxRecursionDepth = Link::MAX_RECURSION; }

            $oLinkQueue = LinkQueue::GetInstance();

            if($oLinkQueue->Count() == 0) {
                $oLink = new Link($sLinkParent, $sLinkParent);
                $oLinkQueue->Enqueue($oLink);
            }

            while($oLink = $oLinkQueue->Dequeue()) {
                $oLink->Crawl();
            }
        }
    }

    abstract class LinkRule {
        abstract public function IsValid(Link $oLink);
        final public function Bitwise() {
            $aClass = array_values(array_filter(get_declared_classes(), create_function('$v', 'return get_parent_class($v) == "'.get_parent_class($this).'";')));
            $sClass = get_class($this);
            $nClass = array_search($sClass, $aClass);
            return pow(2, (int)$nClass+1);
        }

        public function __toString() {
            return (string)$this->Bitwise();
        }
    }

    class StrictDomain extends LinkRule {
        public function IsValid(Link $oLink) {

            $sLinkChild = $oLink->sLink;
            $sLinkParent = $oLink->sLinkParent;

            $bLinkChild = preg_match_all('/(https?:\/\/)?(.[^\/]+\/?)?(.+\/?)*/', $sLinkChild, $aLinkChild);
            $bLinkParent = preg_match_all('/(https?:\/\/)?(.[^\/]+\/?)?(.+\/?)*/', $sLinkParent, $aLinkParent);

            return (($aLinkChild[2][0] == $aLinkParent[2][0]) ? true : false);  
        }
    }

    class ChildDomain extends LinkRule {
        public function IsValid(Link $oLink) {

            $sLinkChild = $oLink->sLink;
            $sLinkParent = $oLink->sLinkParent;

            $bLinkChild = preg_match_all('/(https?:\/\/)?(.[^\/]+\/?)?(.+\/?)*/', $sLinkChild, $aLinkChild);
            $bLinkParent = preg_match_all('/(https?:\/\/)?(.[^\/]+\/?)?(.+\/?)*/', $sLinkParent, $aLinkParent);

            if(eregi('^(.(s|p)?html|.phps?|.aspx?|.cf)', $aLinkChild[2][0])) { return false; }
        }
    }

?>